package com.jiuzhou.crawlers.taobao.pipeline.taobao;

import java.util.Map;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.log4j.Logger;

import com.jiuzhou.crawlers.taobao.processor.taobao.ItemDetailTaoBaoProcessor;
import com.jiuzhou.crawlers.taobao.util.HdfsUtil;

import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

public class TaobaoCommendsPipeline implements Pipeline{
    private String filePath;//hadoop文件夹路径
    public static StringBuffer goodsStringBuffer = new StringBuffer();//商品信息缓存字符串
    private int numPerFile ;
    private static  org.apache.log4j.Logger log = Logger.getLogger(TaobaoItemDetailPipeline.class);
    
	/** 爬虫持久化类构造函数，传入文件路径
	 * @param filePath
	 */
	public TaobaoCommendsPipeline(String filePath, int numPerFile){
		this.filePath =filePath;
		this.numPerFile = numPerFile;
	}
    
	@Override
	public synchronized  void process(ResultItems resultItems, Task arg1) {
		if (!this.filePath.endsWith("/"))
			this.filePath += "/";
		//输出page.putFiel中的所有键值对
		
		int commentsTotal = Integer.parseInt(resultItems.get("total").toString());
		if(commentsTotal>0 ){
			for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
				if(!entry.getKey().equals("total")){
					goodsStringBuffer.append(entry.getKey() + ":\t" + entry.getValue() + "\n");
				}
			
			}
		}
		ItemDetailTaoBaoProcessor.flag++;
		//为存储的文件取名，名字为搜索的关键词
		String url = resultItems.getRequest().getUrl().toString();
	   //String fileName = url.substring(url.indexOf("#")+1,url.length());
      //  int  fileLineLength = ItemDetailTaoBaoProcessor.brandLineCount.get(fileName).size();
        System.out.println("完成链接提取---"+url +"        flag="+ItemDetailTaoBaoProcessor.flag);
        	//if(ItemDetailTaoBaoPreocessor.flag == ItemDetailTaoBaoPreocessor.brandLineCount.get(fileName).get(ItemDetailTaoBaoPreocessor.line)){
             if(ItemDetailTaoBaoProcessor.flag ==this.numPerFile){
        		//String fileType = fileName.substring(fileName.indexOf("+")+1,fileName.length());
    			String path = this.filePath+"/comment/"+DigestUtils.md5Hex(resultItems.getRequest().getUrl());
        		HdfsUtil.WriteToHdfs(path,goodsStringBuffer.toString());
        	  	goodsStringBuffer.delete(0,goodsStringBuffer.length()-1);
        	  	System.out.println("完成爬取，文件名为"+path);
        	  	ItemDetailTaoBaoProcessor.flag=0;   
             }
        }
		
	

}
